package com.spark;

import cn.edu.hfut.dmic.contentextractor.ContentExtractor;
import cn.edu.hfut.dmic.contentextractor.News;
import cn.edu.hfut.dmic.webcollector.net.HttpRequest;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.io.FileUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Created by TRS on 2017/6/19.
 */
public class Html {

    public static void main(String[] args) throws Exception {

        Map<Integer, String> types = new HashMap<Integer, String>();
        types.put(0, "http://auto.163.com/");
        types.put(1, "http://money.163.com/");
        types.put(2, "http://tech.163.com/");
        types.put(3, "http://jiankang.163.com/");
        types.put(4, "http://sports.163.com/");
        types.put(5, "http://travel.163.com/");
        types.put(6, "http://edu.163.com/");
        types.put(7, "http://hr.163.com/position/list.do?postType=&currentPage=1");
        types.put(8, "http://book.163.com/special/00923KMN/shuxun_more.html");
        types.put(9, "http://war.163.com/");

        List<String> lines = new ArrayList<String>();


        for (Map.Entry<Integer, String> en : types.entrySet()) {
            try {

                List<String> urls = getUrls(en.getValue());
                System.out.println(urls.size());

                for (String u : urls) {

                    System.out.println(u);

                    try{
                        News news = getNews(u);
                        JSONObject json = new JSONObject();
                        json.put("title", news.getTitle());
                        json.put("content", getTextFromTHML(news.getContent()));
                        json.put("time", news.getTime());
                        json.put("url", news.getUrl());
                        json.put("type", en.getKey());
                        lines.add(json.toJSONString());

                    }catch (Exception e){

                    }


                }

            } catch (Exception e) {

            }
        }

        File f = new File("E:\\file\\1.json");
        FileUtils.writeLines(f, "UTF-8", lines);

    }

    private static String getTextFromTHML(String html) {
        Document doc = Jsoup.parse(html);
        String text1 = doc.text();
        // remove extra white space
        StringBuilder builder = new StringBuilder(text1);
        int index = 0;
        while (builder.length() > index) {
            Character tmp = builder.charAt(index);
            if (Character.isSpaceChar(tmp) || Character.isWhitespace(tmp)) builder.setCharAt(index, ' ');
            index += 1;
        }
        text1 = builder.toString().replaceAll(" +", " ").trim();

        return text1;
    }


    public static News getNews(String url) throws Exception {
        Document doc = Jsoup.connect(url).get();
        String html=doc.html();

        News news = ContentExtractor.getNewsByHtml(html, url);
        return news;
    }

    public static List<String> getUrls(String url) throws IOException {
        Document doc = Jsoup.connect(url).get();

        Element content = doc.body();
        Elements links = content.getElementsByTag("a");
        List<String> urls = new ArrayList<String>();
        for (Element link : links) {
            String linkHref = link.attr("abs:href");
            String linkText = link.text();
            urls.add(linkHref);
        }
        return urls;
    }
}
